rm(list = ls())

source('param_general.R')
keepVars <- c('sys_recid','maskssn','faclnbr','admitdate','dischdate','age','zipcode','atten_phyid','oper_phyid','msdrg','yearDat','agedays','sys_recid','qtr','white','black','hispanic','payer','charges','fac_region','ptcounty')
keepVarsCharls <- c('sys_recid','prindiag',paste0('othdiag',1:30),'poa_prin_diag',paste0('poa',1:30),'yearDat')

readFileFunc <- function(yr){

    fileNm <- ifelse(yr %in% seq(2006,2011),
                     paste0(rawDatDir,substr(as.character(yr),3,4),'_In.dat'),
                     paste0(rawDatDir,yr, '_IN.dat'))
    dat <- fread(fileNm,integer64 = 'character')

    setnames(dat,names(dat),tolower(names(dat)))
    setnames(dat, 'year', 'yearDat')
    
    if (yr < 2010){
        dat[gender == 2 ,sex := "F"]
        setnames(dat, 'drg','msdrg')
        setnames(dat,c('attenphyid','operphyid'),c('atten_phyid','oper_phyid'))
        dat[,black := race == 3 | race == 6]
        dat[,white := race == 4 | race == 5]
        dat[,hispanic := race == 5 | race == 6]
    }else{
        dat[,black := race == 3]
        dat[,white := race == 4 | race == 5]
        dat[,hispanic := ethnicity == "E1"]
    }
    dat[,charges := rowSums(.SD)-tchgs,.SDcols = grep('chgs',names(dat))]
   
    return(dat)
}


processFileFunc <- function(dat){  
    useDat <- dat[!maskssn %in% c("Nzc3Nzc3Nzc3","MDAwMDAwMDAw", "NTU1NTU1NTU1") &
                  ptstate == "FL" &
                  sex == "F" &
                  ifelse(yearDat ==2006 |(yearDat == 2007 & qtr !=4), msdrg %in% seq(370L,375L),msdrg %in% c(seq(765,768),774,775)),
                  keepVars, with = FALSE]
    useDatCharls <- dat[sys_recid %in% useDat$sys_recid,
                        keepVarsCharls,with=FALSE]

    
    useDat[,payer := factor(payer)]
    levels(useDat$payer) <- list(Medicare = c('A','B'), Medicaid = c('C'), MedicaidMCO = 'D', Commercial = c('E','F','G'), FedGovt = c('I'), VA = 'J', OthGovt = 'K', SelfCharity = c('L','N'), Other = c('M','O'))
    return(list(useDat = useDat, useDatCharls = useDatCharls))
}

combineFunc <- function(yr){
    dat <- readFileFunc(yr)
    useDatBoth <- processFileFunc(dat)
    return(useDatBoth)
}

useDatBothList <- mclapply(seq(2006,2014), combineFunc,mc.cores = 10) 

useDatList <- list()
useDatCharlsList <- list()
for(y in 1:length(useDatBothList)){
    useDatList[[y]] <- useDatBothList[[y]][['useDat']]
    useDatCharlsList[[y]] <- useDatBothList[[y]][['useDatCharls']]
}
    
useDatAll <- rbindlist(useDatList)
useDatCharlsAll <- rbindlist(useDatCharlsList)

save(useDatAll, file = paste0(datDir,'base_ld_women.RData'))
write.csv(useDatCharlsAll,file = paste0(datDir,'diagCharls_ld_women.csv'))


 




